/**
 * 
 */
package com.sofast.engine;

import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;

/**
 * @author alex
 *
 */
public class Spider {
	
	private String url;
	private String cs;
	
	public Spider(String url, String cs) {
		
		this.url = url;
		this.cs = cs;
	}

	public String getUrl() {
		return url;
	}

	public String getCs() {
		return cs;
	}
	
	// __ Private Method
	public String getHtmlString() {
		StringBuffer sb = new StringBuffer();
		String tempString;
		URL urlc;
		try {
			urlc = new URL(url);
			HttpURLConnection connection = (HttpURLConnection)urlc.openConnection();
			connection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; " +
					"Windows NT; DigExt)");
			BufferedReader in = new BufferedReader(
					new InputStreamReader(connection.getInputStream(),cs));
			while((tempString = in.readLine()) != null) {
				
				tempString = tempString.replaceAll("&amp;", "&")
				.replaceAll("\\&[a-zA-Z]{1,10};", "");
				//.replaceAll(" ", "");
				sb.append(tempString);
				//System.out.println(tempString);
				
			}
			in.close();
			
		} catch (Exception e) {
			e.printStackTrace();
		}
		return sb.toString().trim();
	}
	
	public static void downloadImgae(String imageURL, String imageName) {
		InputStream in = null;
		OutputStream out = null;
		
		try {
			URL url = new URL(imageURL);
			in = url.openStream();
			
			out = new FileOutputStream(imageName);
			
			byte[] buffer = new byte[4096];
			int bytes_read;
			while ((bytes_read = in.read(buffer)) != -1) {
				out.write(buffer, 0, bytes_read);
			}
			
			in.close();
			out.close();
		} catch (Exception e) {
			e.printStackTrace();
		} 
	}
	
	public static void main(String[] args) {
		String url = "http://www.1ting.com/group/group0_1.html";
		Spider spider = new Spider(url,"gbk");
		spider.getHtmlString();
	}

}
